package com.estock.crawler;

import java.io.UnsupportedEncodingException;
import java.util.regex.Pattern;

import org.apache.http.HttpStatus;

import com.estock.common.util.FileUtil;
import com.estock.crawler.utils.ESCrawlerUtil;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class JqkaCrawler extends WebCrawler {

	private final static Pattern FILTERS = Pattern
			.compile(".*(\\.(css|js|bmp|gif|jpe?g|ico"
					+ "|png|tiff?|mid|mp2|mp3|mp4"
					+ "|wav|avi|mov|mpeg|ram|m4v|pdf"
					+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

	private String skippedPageFile;
	private String errorPageFile;
	
	public JqkaCrawler() {
		skippedPageFile = get10jqkaDataDir() + "skippedPage.txt";
		errorPageFile = get10jqkaDataDir() + "errorPage.txt";
	}
	
	private String get10jqkaDataDir() {
		return ESCrawlerUtil.get10jqkaRawDataDir();
	}
	
	@Override
	public boolean shouldVisit(Page referringPage, WebURL url) {
		String href = url.getURL().toLowerCase() + "\n";
		if (FILTERS.matcher(href).matches() ||
				!href.startsWith(ESCrawlerUtil.DOMIN_10jqka)) {
			FileUtil.writeBytes(skippedPageFile, href.getBytes(), true);
			return false;
		}

		return true;
	}

	/**
	 * This function is called when a page is fetched and ready to be processed
	 * by your program.
	 */
	@Override
	public void visit(Page page) {
		String url = page.getWebURL().getURL();

		if (page.getParseData() instanceof HtmlParseData) {
			HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
			String html = htmlParseData.getHtml();
			String filename = generateFilename(url);
			
			try {
				FileUtil.writeBytes(filename, html.getBytes(page.getContentCharset()), false);
			} catch (UnsupportedEncodingException e) {
				logger.info("UnsupportedEncodingException: " + e);
			}
		}
	}
	
	@Override
	protected void handlePageStatusCode(WebURL webUrl, int statusCode, String statusDescription) {
		if (statusCode != HttpStatus.SC_OK) {
			String href = webUrl.getURL().toLowerCase() + "\n";
			FileUtil.writeBytes(errorPageFile, href.getBytes(), true);
			if (statusCode == HttpStatus.SC_NOT_FOUND) {
				logger.warn("Broken link: {}, this link was found in page: {}", webUrl.getURL(), webUrl.getParentUrl());
			} else {
				logger.warn("Non success status for link: {} status code: {}, description: ", webUrl.getURL(), statusCode,
						statusDescription);
			}
	    }
	}

	private String generateFilename(String url) {
		// remove http:// prefix
		String filename = get10jqkaDataDir() + url.replace("http://", "");
		// remove / if it is the last char
		if (filename.endsWith("/")) {
			filename = filename.substring(0, filename.length() - 1);
		}
		if (!filename.endsWith(".html")) {
			filename += ".html";
		}
		
		return filename;
	}
}